{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# extract data --> four category\n",
    "## remove other category data\n",
    "- angry : 0        ang(1103)\n",
    "- happy : 1       exc(1041), hap(595)\n",
    "- sad : 2          sad(1084)   \n",
    "- neutral : 3     neu(1708)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_path = 'four_category'\n",
    "# target_path = 'four_category_G'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_label = []\n",
    "with open('../data/processed/IEMOCAP/processed_label.txt') as f:\n",
    "    full_label = f.readlines()\n",
    "full_label = [ x.strip() for x in full_label]    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 01 ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "tmp_ids = []\n",
    "with open('../data/processed/IEMOCAP/label.csv') as f:\n",
    "    csv_reader = csv.reader(f)\n",
    "    tmp_ids = [x for x in csv_reader]\n",
    "    tmp_ids = [x[0] for x in tmp_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/processed/IEMOCAP/ordered_ids.txt', 'w') as f:\n",
    "    for _id in tmp_ids:\n",
    "        f.write(_id+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open('../data/processed/IEMOCAP/ordered_ids.txt') as f:\n",
    "    data = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from file_util import *\n",
    "create_folder('../data/processed/IEMOCAP/'+target_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/processed/IEMOCAP/' + target_path + '/FC_ordered_ids.txt', 'w') as f:\n",
    "    for i, label in enumerate(full_label):\n",
    "        if label != '-1':\n",
    "            f.write( data[i] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 02 label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open('../data/processed/IEMOCAP/processed_label.txt') as f:\n",
    "    data = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/processed/IEMOCAP/' + target_path + '/FC_label.txt', 'w') as f:\n",
    "    for i, label in enumerate(full_label):\n",
    "        if label != '-1':            \n",
    "            f.write( data[i] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 03 MFCC_sequenceN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "# with open('../data/processed/IEMOCAP/processed_MFCC12EDAZ_sequenceN.txt') as f:\n",
    "with open('../data/processed/IEMOCAP/processed_MFCC12EDA_sequenceN.txt') as f:\n",
    "    data = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('../data/processed/IEMOCAP/four_category/FC_MFCC12EDAZ_sequenceN.txt', 'w') as f:\n",
    "with open('../data/processed/IEMOCAP/' + target_path + '/FC_MFCC12EDA_sequenceN.txt', 'w') as f:\n",
    "    for i, label in enumerate(full_label):\n",
    "        if label != '-1':\n",
    "            f.write( data[i] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 04 prodody"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10039, 35)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "data = np.load('../data/processed/IEMOCAP/processed_prosody.npy')\n",
    "np.shape(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_num = 5531\n",
    "new_data = np.zeros([5531, 35], dtype=np.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "for i, label in enumerate(full_label):\n",
    "    if label != '-1':\n",
    "        new_data[index] = data[i]\n",
    "        index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/processed/IEMOCAP/' + target_path + '/FC_prodosy.npy', new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 05 emobase2010"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10039, 1582)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "data = np.load('../data/processed/IEMOCAP/processed_emobase2010.npy')\n",
    "np.shape(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_num = 5531\n",
    "new_data = np.zeros([5531, 1582], dtype=np.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "for i, label in enumerate(full_label):\n",
    "    if label != '-1':\n",
    "        new_data[index] = data[i]\n",
    "        index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/processed/IEMOCAP/four_category/FC_emobase2010.npy', new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 06 MFCC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10039, 750, 39)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# data = np.load('../data/processed/IEMOCAP/processed_MFCC12EDAZ.npy')\n",
    "data = np.load('../data/processed/IEMOCAP/processed_MFCC12EDA.npy')\n",
    "np.shape(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_num = 5531\n",
    "new_data = np.zeros([5531, 750, 39], dtype=np.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "for i, label in enumerate(full_label):\n",
    "    if label != '-1':\n",
    "        new_data[index] = data[i]\n",
    "        index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.save('../data/processed/IEMOCAP/four_category/FC_MFCC12EDAZ.npy', new_data)\n",
    "np.save('../data/processed/IEMOCAP/' + target_path + '/FC_MFCC12EDA.npy', new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 06-A transcription"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10039"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "data = []\n",
    "with open('../data/processed/IEMOCAP/processed_tran.csv') as f:\n",
    "    csv_reader = csv.reader(f)\n",
    "    data = [ x[1] for x in csv_reader ]\n",
    "len(data)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/processed/IEMOCAP/' + target_path + '/FC_trans.txt', 'w') as f:\n",
    "    for i, label in enumerate(full_label):\n",
    "        if label != '-1':\n",
    "            f.write( data[i] + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 06-B transcription - google speech api  - use it when ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import csv\n",
    "\n",
    "# data = []\n",
    "# with open('../data/processed/IEMOCAP/processed_tran_fromG.csv') as f:\n",
    "#     csv_reader = csv.reader(f)\n",
    "#     data = [ x[1] for x in csv_reader ]\n",
    "# len(data)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('../data/processed/IEMOCAP/' + target_path + '/FC_trans.txt', 'w') as f:\n",
    "#     for i, label in enumerate(data):\n",
    "#         f.write( data[i] + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 07-A transcription NUMPY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10039, 128)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from nlp_util import *\n",
    "import pickle\n",
    "\n",
    "data = np.load('../data/processed/IEMOCAP/processed_trans.npy')\n",
    "np.shape(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_num = 5531\n",
    "new_data = np.zeros([5531, 128], dtype=np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "for i, label in enumerate(full_label):\n",
    "    if label != '-1':\n",
    "        new_data[index] = data[i]\n",
    "        index += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('../data/processed/IEMOCAP/four_category/FC_trans.npy', new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's test\n",
    "dic = {}\n",
    "with open('../data/processed/IEMOCAP/dic.pkl') as f:\n",
    "    dic = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['yes', ',', 'i', 'did', 'it', '.', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_', '_PAD_']\n"
     ]
    }
   ],
   "source": [
    "inv_dic = create_invert_dic(dic)\n",
    "index_to_sentence(inv_dic, new_data[4999])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### 07-B transcription NUMPY - use it when ASR-trans is ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "# from nlp_util import *\n",
    "# import pickle\n",
    "\n",
    "# data = np.load('../data/processed/IEMOCAP/processed_trans_G.npy')\n",
    "# np.shape(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# total_num = 5531\n",
    "# new_data = np.zeros([5531, 128], dtype=np.int)\n",
    "\n",
    "# for i, label in enumerate(data):\n",
    "#     new_data[i] = data[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.save('../data/processed/IEMOCAP/' + target_path + '/FC_trans.npy', new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # let's test\n",
    "# dic = {}\n",
    "# with open('../data/processed/IEMOCAP/dic_G.pkl') as f:\n",
    "#     dic = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inv_dic = create_invert_dic(dic)\n",
    "# index_to_sentence(inv_dic, new_data[4999])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf14_p27]",
   "language": "python",
   "name": "conda-env-tf14_p27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
